from transformers import AutoModel, AutoTokenizer, AutoModelForSequenceClassification
from transformers import pipeline
from torch.nn.functional import softmax

# checkpoint = 'E:\\ai\\huggingface\\models\\distilbert-base-uncased-finetuned-sst-2-english'
checkpoint = 'E:\\ai\\huggingface\\models\\TinyBERT_General_4L_312D'

raw_input = "I've been waiting for a HuggingFace course my whole life."

print('-'*50)

tokenizer = AutoTokenizer.from_pretrained(checkpoint)
inputs = tokenizer(raw_input)
tokens = tokenizer.tokenize(raw_input)
print(inputs)
print(tokens)

ids = tokenizer.convert_tokens_to_ids(tokens)
print(ids)

decoded_ = tokenizer.decode(ids)
print(decoded_)
